package regex;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 网络爬虫-取链接 wget
 */
public class WebSpiderTest {
    /**
     * 获得urlStr对应的网页源码内容
     * @param urlStr
     * @return
     */
    public static String getURLContent(String urlStr,String charset){
        StringBuilder sb=new StringBuilder();
        try {
            URL url=new URL(urlStr);

            BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream(), Charset.forName(charset)));
            String temp="";
            while ((temp=br.readLine())!=null){
                sb.append(temp);
                sb.append("\r\n");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sb.toString();
    }

    /**
     *
     * @param destStr
     * @param regexStr
     * @return
     */
    public static List<String> getMatcherStrs(String destStr,String regexStr){
        //Pattern p=Pattern.compile("<a[\\s\\S]+?</a>");//取到整个网页中的整个超链接
        Pattern p=Pattern.compile(regexStr);//取到整个网页中的整个超链接的地址
        Matcher m=p.matcher(destStr);
        List<String> result=new ArrayList<String>();
        while (m.find()){
            result.add(m.group(1));
        }
        return result;
    }
    public static void main(String[] args) {
        String destStr= getURLContent("https://www.163.com/","gbk");
        //System.out.println(destStr);
        List<String> result=getMatcherStrs(destStr,"href=\"([\\w\\s./:]+?)\"");
        for(String temp:result){
            System.out.println(temp);
        }
    }
}
